Source code for hysop.tools.profiler

# Copyright (c) HySoP 2011-2024
#
# This file is part of HySoP software.
# See "https://particle_methods.gricad-pages.univ-grenoble-alpes.fr/hysop-doc/"
# for further info.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


"""
Tools to collect time profiling information
for hysop classes.
"""
from hysop.core.mpi import main_rank
from hysop.tools.numpywrappers import npw
from hysop.tools.units import time2str
import numpy as np


[docs] class FProfiler: """ Class for time measurments on the fly. The objects can be linked to a class method. """ def __init__(self, fname): """Object to profile on the fly sections of code, methods ... Usage: >>> from hysop.tools.profiler import FProfiler >>> from hysop.core.mpi import Wtime as ftime >>> prof = FProfiler('some_name') >>> start = ftime() >>> # do something ... [1] >>> prof += ftime() - start >>> # do something else ... [2] >>> prof += ftime() - start >>> # ... >>> # print(prof) >>> # --> display total time spent in do [1] and [2] >>> # and number of calls of prof """ # Function name self.fname = fname # Total execution time self.total_time = 0.0 # Number of calls of the fprofiler self.nb_calls = 0
[docs] def get_name(self): """Profiler name""" return self.fname
[docs] def __iadd__(self, t): """+= operator""" self.total_time += t self.nb_calls += 1 return self
def __str__(self): if self.nb_calls > 0: s = "{} ncalls={}, total={}, mean={}".format( self.fname, self.nb_calls, time2str(self.total_time), time2str(self.total_time / self.nb_calls), ) else: s = "" return s
[docs] def get(self): if self.nb_calls > 0: return ( self.fname, self.nb_calls, time2str(self.total_time), time2str(self.total_time / self.nb_calls), ) else: return ("", 0, None, None)
[docs] class Profiler: """ Object used to collect profiling information inside operators. """ def __init__(self, obj): """ Collect profiling information for all operator method decorate with @profile. Parameters ---------- obj : object (python class) instance. See requirements in notes below. Notes: * obj must have '_get_profiling_info' and 'name' attribute/method. """ self.summary = {} self.table = [] # profiled object self._obj = obj _comm = self.get_comm() _comm_size = _comm.Get_size() # A dictionnary of profiled functions/methods as keys # and elapsed time as value. self._elems = {} self._l = 1 self.all_data = None self.node_id = None
[docs] def down(self, l): self._l = l + 1
[docs] def get_name(self): """Return the name of the profiled object""" _name = self._obj.name return _name
[docs] def get_comm(self): """Return the communicator associated to the profiled object""" _comm = self._obj.mpi_params.comm return _comm
[docs] def __iadd__(self, other): """+= operator. Append a new profiled function to the collection""" if not other.get_name() in self._elems.keys(): self._elems[str(other._obj)] = other return self
def __setitem__(self, key, value): self._elems[key] = value def __getitem__(self, item): try: return self._elems[item] except KeyError: self._elems[item] = FProfiler(item) return self._elems[item] def __str__(self): summary = self.summary if len(summary) > 0: if ( (self._l > 1) and (len(summary) == 1) and isinstance(next(iter(summary.values())), FProfiler) ): s = f">{self.get_name()}::{next(iter(summary.values()))}" else: s = "{}[{}]>{}{}".format( "\n" if (self._l == 1) else "", main_rank, self.get_name(), " profiler report" if (self._l == 1) else "", ) for ( v ) in ( summary.values() ): # sorted(summary.values(), key=lambda x: x.total_time): if len(str(v)) > 0: s += "\n{}".format(" " * self._l + str(v)) else: s = "" return s
[docs] def write(self, prefix="", hprefix="", with_head=True): """ Parameters ---------- prefix : string, optional hprefix : string, optional with_head : bool, optional """ if prefix != "" and prefix[-1] != " ": prefix += " " if hprefix != "" and hprefix[-1] != " ": hprefix += " " if self._comm.Get_rank() == 0: s = "" h = hprefix + "Rank" for r in range(self._comm_size): s += prefix + f"{r}" for i in range(len(self.all_names)): s += f" {self.all_times[i][r]}" s += "\n" s += prefix + "-1" for i in range(len(self.all_names)): h += " " + self.all_names[i] s += f" {self.all_times[i][self._comm_size]}" h += "\n" if with_head: s = h + s print(s)
[docs] def summarize(self): """ Update profiling values and prepare data for a report with print or write. """ # reset summary self.summary = {} # collect profiling results from decorated object(s), if any. self._obj._get_profiling_info() from hysop.fields.continuous_field import Field i = 0 # Recursive summarize for k in self._elems.keys(): try: # Either elem[k] is a FProfiler ... self.summary[self._elems[k].total_time] = self._elems[k] except AttributeError: # ... or a Profiler i += 1 self._elems[k].down(self._l) self._elems[k].summarize() if isinstance(self._elems[k]._obj, Field): self.summary[1e10 * i] = self._elems[k] else: self.summary[1e8 * i] = self._elems[k] # Flatten elements self.table = [] for k in sorted(self._elems.keys()): if isinstance(self._elems[k], FProfiler): tt = ( str(self._obj) + "." + k, self._elems[k].total_time, self._elems[k].nb_calls, ) self.table.append(tt) for k in sorted(self._elems.keys()): if isinstance(self._elems[k], Profiler): for e in self._elems[k].table: tt = (str(self._obj) + "." + e[0], e[1], e[2]) self.table.append(tt)
[docs] def tasks_summarize(self): """Collect all profiling data across processes.""" # Table content is (name, time, ncalls) if self._l == 1: all_data = self.all_data comm = self.get_comm() rk = comm.Get_rank() comm_size = comm.Get_size() nb = comm.allgather(len(self.table)) all_names = comm.allgather([_[0] for _ in self.table]) all_times = comm.allgather([_[1] for _ in self.table]) all_calls = comm.allgather([int(_[2]) for _ in self.table]) all_data = {} # all_data structure : task_size, calls nb, self total, task mean, task min, task max nelem = 6 for r in range(comm_size): for n, t, c in zip(all_names[r], all_times[r], all_calls[r]): if n not in all_data: all_data[n] = [ 0, ] * nelem + [ [], ] all_data[n][0] += 1 all_data[n][1] += c if r == rk: all_data[n][2] = t all_data[n][nelem].append(t) for n in all_data.keys(): all_data[n][1] //= all_data[n][0] all_data[n][3] = npw.average(all_data[n][nelem]) all_data[n][4] = npw.min(all_data[n][nelem]) all_data[n][5] = npw.max(all_data[n][nelem]) self.all_data = all_data